/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package pxgrid_controllers;

import java.io.*;
import java.text.SimpleDateFormat;
import java.util.*;
import javax.xml.parsers.*;
import javax.xml.transform.*;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;
import org.w3c.dom.*;

import pxgrid_entities.*;
import pxgrid_utils.*;

/**
 * PXXGridController
 * A class to handle interaction with an Xgrid (Xgrid extension of PXGridcontroller)
 * 
 * @author Mark Bate
 */
public class PXXGridController extends PXGridController {

    //defines whether to print debug messages or not.
    private Boolean debug;
    
    /**
     * create a new instance of the PXXGridController
     * @param debug Boolean setting as true will turn debug outputs on
     */
    public PXXGridController(Boolean debug) {
        this.debug = debug;
    }

    /**
     * Takes a PXSubmission loaded with PXJobs, and submits them to the xgrid.
     * The xgrid will return a job id, which this method then assigns as the 
     * xgrid_job_id for each PXJob. The jobs are then reloaded ito the 
     * original PXSubmission, and the PXSubmission is returned.
     * Essentially it calls the writeSubmissionFileForJob() and the 
     * submitSubmissionFileToGrid() for each PXJob.
     * 
     * Note: each PXJob must have it's:
     * "grid_controller", "job_name", "input_mtz", "input_pdb", "space_group",
     * "f_label", "sigf_label", "molecular_weight", 
     * "number_of_molecules_in_asymmetric_unit", 
     * "number_of_copies_of_ensemble_to_search_for", "rmsd" and "packing"
     * variables set.
     * The PXSubmission must also have it's directory_path and submission_ID variables
     * set.
     *
     * @param submission PXSubmission
     * @return PXSubmission the same PXSubmission as was passed in, with each PXJobs'
     * grid_job_id value set
     * @throws java.lang.Exception thrown if any of the above required values 
     * are not set, or if the PXSubmission does not contain any PXJobs.
     */
    @Override
    public PXSubmission submitJobsToGrid(PXSubmission submission) throws Exception {

        //DEBUG STATEMENT
        if(this.debug) {
            System.out.println("PXXGridController:submitJobsToGrid()# BEGIN");
        }
        
        //DEBUG STATEMENT
        if(this.debug) {
            System.out.println("PXXGridController:submitJobsToGrid()# retreive ArrayList of PXJobs from PXSubmission");
        }
        
        //grab the jobs from the submission
        ArrayList jobs = submission.getJobs();
        
        //check that it's not empty
        if(jobs.isEmpty()) {
            throw new Exception("submission does not appear to contain any jobs");
        }
        
        //DEBUG STATEMENT
        if(this.debug) {
            System.out.println("PXXGridController:submitJobsToGrid()# iterate through PXJobs");
        }
        
        //iterate through the PXJob[] array
        for (int i = 0; i < jobs.size(); i++) {
            
            PXJob job = (PXJob)jobs.get(i);
            
            
            //DEBUG STATEMENT
            if(this.debug) {
                System.out.println("PXXGridController:submitJobsToGrid()# check PXJob has variables set");
            }
            
            
            //check that we have the variables we want are set and valid
            
            if(job.getJobName() == null) {
                throw new Exception("job_name not set for PXJob "+i);
            }
            
            if(job.getGridController() == null) {
                throw new Exception("grid_controller not set for PXJob "+i);
            }
            
            if(job.getInputMTZ() == null) {
                throw new Exception("input_mtz not set for PXJob "+i);
            }
            
            if(job.getInputPDB() == null) {
                throw new Exception("input_pdb not set for PXJob "+i);
            }
            
            if(job.getSpaceGroup() == null) {
                throw new Exception("space_group not set for PXJob "+i);
            }
            
            if(job.getFLabel() == null) {
                throw new Exception("f_label not set for PXJob "+i);
            }
            
            if(job.getSigfLabel() == null) {
                throw new Exception("sigf_label not set for PXJob "+i);
            }
            
            if(job.getMolecularWeight() <= 0) {
                throw new Exception("molecular_weight invalid for PXJob "+i);
            }
            
            if(job.getNumberOfMoleculesInAsymmetricUnit() <= 0) {
                throw new Exception("number_of_molecules_in_asymmetric_unit invalid for PXJob "+i);
            }
            
            if(job.getNumberOfCopiesOfEnsembleToSearchFor() <= 0) {
                throw new Exception("number_of_copies_of_ensemble_to_search_for invalid for PXJob "+i);
            }
            
            if(job.getRMSD() <= 0) {
                throw new Exception("rmsd invalid for PXJob "+i);
            }
            
            if(job.getPacking() <= 0) {
                throw new Exception("packing invalid for PXJob "+i);
            }
            
            //if we've made it to here everything should be good.
            
            
            //DEBUG STATEMENT
            if(this.debug) {
                System.out.println("PXXGridController:submitJobsToGrid()# PXJob has variables set");
            }
            
            
            //try writing the XML Plist file for the PXJob and then submitting it
            try {
                
                //DEBUG STATEMENT
                if(this.debug) {
                    System.out.println("PXXGridController:submitJobsToGrid()# writeSubmissionFileForJob()");
                }
                
                //write the submission file for the job
                String submission_file = this.writeSubmissionFileForJob(job, submission.getDirectoryPath()+"/"+submission.getSubmissionID());
                
                //DEBUG STATEMENT
                if(this.debug) {
                    System.out.println("PXXGridController:submitJobsToGrid()# submitSubmissionFileToGrid()");
                }
                
                //submit it to the grid, will give us a job_id
                int job_id = this.submitSubmissionFileToGrid(submission_file, job.getGridController());
                
                //DEBUG STATEMENT
                if(this.debug) {
                    System.out.println("PXXGridController:submitJobsToGrid()# set the id back from the grid");
                }
                
                //set the grid_job_id of the PXJob
                job.setGridJobID(job_id);
                
            } catch (Exception ex) {
                ex.printStackTrace();
            }
        }

        //DEBUG STATEMENT
        if(this.debug) {
            System.out.println("PXXGridController:submitJobsToGrid()# END");
        }

        //return the PXSubmission
        return submission;
        
    } //end of method.

    
    /**
     * Gets the status of a job from the Xgrid
     * @deprecated This method has been replaced by getJobAttributes() methods,
     * which provides more indepth information about the job.
     * 
     * @param job_id int the id (assigned by the grid) of the job you wish to get 
     * status for
     * @param grid_name String the display_name of the Xgrid to query
     * @return String the status of the job.
     * @throws java.lang.Exception
     */
    @Override
    public String getJobStatus(int job_id, String grid_name) throws Exception {
        
        //check the job_id is valid, ie. we shouldn't have negative job ids,
        //if we do, something is very very wrong. most likely Steve's fault ;p
        if(job_id <= 0) {
            throw new Exception("job_id must be an integer greater than or equal to zero");
        }
        
        //check that the grid name isn't null.
        if(grid_name == null ) {
            throw new Exception("grid_name must be set");
        }
        
        //parse the xgrid_definitions.xml file into a Map
        Map gridMap = PXUtils.parseGridXML("/mrconfig/xgrid_definitions.xml");
        
        //convert gridMap entry into a PXGrid entity.
        PXGrid grid = (PXGrid) gridMap.get(grid_name);

        //get the various attributes required to submit to the grid
        String host = grid.getHost();
        String auth_method = grid.getAuth_method();
        String password = grid.getPassword();
        int grid_id = grid.getGridID();
        
        //formulate the command string
        String cmd = "xgrid -h " + host + " -a " + auth_method;

        //if we're authenticating by password, then use it
        if (auth_method.contains("password")) {
            cmd += " -p " + password;
        }

        //add the other stuff to the cmd string
        cmd += " -gid " + grid_id + " -job attributes -id " + job_id;
        
        try {
            //create a Process from the cmd
            Process prcs = Runtime.getRuntime().exec(cmd);
            InputStream cmd_output = prcs.getInputStream();
            
            //wait til it's finished
            int exit_val = prcs.waitFor();

            //hopefully it exited with 0.. 
            if (exit_val == 0) {
                
                //read from the InputStream and put it into BufferedReader
                BufferedReader br = new BufferedReader(new InputStreamReader(cmd_output));
                
                //create a String to readLine() into
                String line;

                //iterate through the BufferedReader & look for "jobStatus"
                while ((line = br.readLine()) != null) {
                    if(line.contains("jobStatus")) {
                        
                        //if we find it, substring() it to just the right word
                        //and return it.
   
                        //hopefully this way is tiger-happy?
                        int begin_split = line.indexOf("=")+1; //this way we get after the =
                        int end_split = line.indexOf(";");
                        
                        return line.substring(begin_split, end_split).trim();
                    }
                }
                
                //close the BufferedReader
                br.close();
            }

        } catch(Exception ex) {
            //prob should do something here?
            ex.printStackTrace();
        }
        
        //the job is now out of radio contact...
        return "Unknown";
    }
    
    
    /**
     * Gets the attributes of a job from the grid.
     * This method replaces the getJobStatus() method.
     * 
     * Note: the PXJob must have it's grid_controller and grid_job_id variables
     * set.
     * 
     * @param job PXJob the job to get the attributes for
     * @return PXJob the same PXJob with values set for status, is_suspended,
     * start_time, stop_time and submission_time, where applicable.
     * @throws java.lang.Exception thrown if the PXJobs grid_controller 
     * and grid_job_id are not set.
     */
    @Override
    public PXJob getJobAttributes(PXJob job) throws Exception {
        
        //check the job_id is valid, ie. we shouldn't have negative job ids,
        //if we do, something is very very wrong. most likely Steve's fault ;p
        if(job.getGridJobID() <= 0) {
            throw new Exception("job_id must be an integer greater than or equal to zero");
        }
        
        //check that the grid name isn't null.
        if(job.getGridController() == null ) {
            throw new Exception("grid_name must be set");
        }
        
        //parse the xgrid_definitions.xml file into a Map
        Map gridMap = PXUtils.parseGridXML("/mrconfig/xgrid_definitions.xml");
        
        //convert gridMap entry into a PXGrid entity.
        PXGrid grid = (PXGrid) gridMap.get(job.getGridController());

        //get the various attributes required to submit to the grid
        String host = grid.getHost();
        String auth_method = grid.getAuth_method();
        String password = grid.getPassword();
        int grid_id = grid.getGridID();
        
        //formulate the command string
        String cmd = "xgrid -h " + host + " -a " + auth_method;

        //if we're authenticating by password, then use it
        if (auth_method.contains("password")) {
            cmd += " -p " + password;
        }

        //add the other stuff to the cmd string
        cmd += " -gid " + grid_id + " -job attributes -id " + job.getGridJobID();
        
        try {
            //create a Process from the cmd
            Process prcs = Runtime.getRuntime().exec(cmd);
            InputStream cmd_output = prcs.getInputStream();
            
            //wait til it's finished
            int exit_val = prcs.waitFor();

            //hopefully it exited with 0.. 
            if (exit_val == 0) {
                
                //read from the InputStream and put it into BufferedReader
                BufferedReader br = new BufferedReader(new InputStreamReader(cmd_output));
                
                //create a String to readLine() into
                String line;

                //iterate through the BufferedReader & look for "jobStatus"
                while ((line = br.readLine()) != null) {
                    if(line.contains("jobStatus") 
                        || line.contains("dateStarted")
                        || line.contains("dateStopped")
                        || line.contains("dateSubmitted")
                        || line.contains("suspended")) {
                        
                        //if we find it, substring() it to just the right word
                        //and return it.
   
                        //hopefully this way is tiger-happy?
                        int begin_split = line.indexOf("=")+1; //this way we get after the =
                        int end_split = line.indexOf(";");
                        
                        String value = line.substring(begin_split, end_split).trim();
                        
                        //set the job status
                        if(line.contains("jobStatus")) {
                            job.setStatus(value);
   
                        //if the job is suspended, set the "isSuspended" value accordingly
                        } else if(line.contains("suspended")) {
                            
                            if(value.equalsIgnoreCase("YES")
                                    || value.equals("1")
                                    || value.equalsIgnoreCase("true")) {
                                job.setIsSuspended(true);
                            } else {
                                job.setIsSuspended(false);
                            }
                            
                        //if we have the dateStarted, set that
                        } else if(line.contains("dateStarted")) {

                            // Note: under Snow Leopard timestamps are quoted
                            // this line removes the quotes
                            value = value.replace("\"", "");

                            //this should be the format that it comes back in..
                            SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd kk:mm:ss Z");
                            //parse it to a Date & set as the Start Time
                            job.setStartTime(sdf.parse(value));
                            
                        } else if(line.contains("dateStopped")) {
                            
                            // Note: under Snow Leopard timestamps are quoted
                            // this line removes the quotes
                            value = value.replace("\"", "");

                            //this should be the format that it comes back in..
                            SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd kk:mm:ss Z");
                            //parse it to a Date & set as the Stop Time
                            job.setStopTime(sdf.parse(value));
                            
                        } else if(line.contains("dateSubmitted")) {

                            // Note: under Snow Leopard timestamps are quoted
                            // this line removes the quotes
                            value = value.replace("\"", "");
                            
                            //this should be the format that it comes back in..
                            SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd kk:mm:ss Z");
                            //parse it to a Date & set as the Submit Time
                            job.setSubmissionTime(sdf.parse(value));
                            
                        }
                    }
                }
                
                //close the BufferedReader
                br.close();
            }

        } catch(Exception ex) {
            //prob should do something here?
            ex.printStackTrace();
        }
        
        //the job is now out of radio contact...
        return job;
    }
    
    
    /**
     * Gets the results for the PXJob from the grid.
     * Note: the PXJob must have it's:
     * grid_controller", "job_name", "input_mtz", "input_pdb" and "space_group"
     * variables set.
     * 
     * @param job PXJob job for which to grab the results back for
     * @param results_dir String the directory to return the files in to
     * @return PXJob the same PXJob, but with the results variables set.
     * @throws java.lang.Exception thrown if any of the above required values are
     * not set
     */
    @Override
    public PXJob getJobResults(PXJob job, String results_dir) throws Exception {
        
        //check if things are set, if not throw exceptions/tantrums
        
        if(job.getJobName() == null) {
            //throw new Tantrum("job_name not set for PXJob ");
            throw new Exception("job_name not set for PXJob ");
        }

        if(job.getGridController() == null) {
            throw new Exception("grid_controller not set for PXJob ");
        }

        if(job.getInputMTZ() == null) {
            throw new Exception("input_mtz not set for PXJob ");
        }

        if(job.getInputPDB() == null) {
            throw new Exception("input_pdb not set for PXJob ");
        }

        if(job.getSpaceGroup() == null) {
            throw new Exception("space_group not set for PXJob ");
        }
        
        if(job.getRMSD() <= 0) {
            throw new Exception("rmsd not set for PXJob ");
        }

        //parse the xgrid_definitions.xml file into a Map
        Map gridMap = PXUtils.parseGridXML("/mrconfig/xgrid_definitions.xml");
        
        //grab the grid_job_id and grid_controller from the PXJob
        int job_id = job.getGridJobID();
        String grid_name = job.getGridController();
        
        //convert gridMap entry into a PXGrid entity.
        PXGrid grid = (PXGrid) gridMap.get(grid_name);

        //get the various attributes required to submit to the grid
        String host = grid.getHost();
        String auth_method = grid.getAuth_method();
        String password = grid.getPassword();
        int grid_id = grid.getGridID();
        
        //formulate the command string
        String cmd = "xgrid -h " + host + " -a " + auth_method;

        //if we're authenticating by password, then use it
        if (auth_method.contains("password")) {
            cmd += " -p " + password;
        }

        //add the other stuff to the cmd string
        cmd += " -gid " + grid_id + " -job results -id " + job_id + " -out " +results_dir;
        
        try {
            
            //create a Process from the cmd
            Process prcs = Runtime.getRuntime().exec(cmd);
            InputStream cmd_stdout = prcs.getInputStream();
            
            //dont wait until it's finished, as it doesn't appear to return an
            //exit code.. or if it does, it takes forever & hangs the browser
            
            //put the InputStream into a BufferedReader
            BufferedReader stdout_br = new BufferedReader(new InputStreamReader(cmd_stdout));
            StringBuilder stdout_sb = new StringBuilder();
            String stdout_line;

            //iterate through the stdout and add to StringBuilder
            while ((stdout_line = stdout_br.readLine()) != null) {
                stdout_sb.append(stdout_line + "<br/>");
            }
            //close the StringBuilder
            stdout_br.close();
            //set the job_stdout
            job.setJobStdout(stdout_sb.toString());
            
            //open the results_dir, and get them as a list
            File results = new File(results_dir);
            String[] results_contents = results.list();
                
            //check that we haven't got an empty directory
            if(results_contents != null) {
                //iterate through the results_contents
                for (int i=0; i<results_contents.length; i++) {
                    
                    //look for files with the input_pdb and space_group in the name
                    if(results_contents[i].contains(job.getInputPDB()) 
                            && results_contents[i].contains(job.getSpaceGroup())
                            && results_contents[i].contains(String.valueOf(job.getRMSD()))) {
                        
                        //check if the file ends in .pdb,
                        //if so then it should be our output_pdb
                        if(results_contents[i].lastIndexOf(".pdb") == (results_contents[i].length()-4)) {
                            job.setOutputPDB(results_contents[i]);
                        }

                        //check if the file ends in .mtz,
                        //if so then it should be our output_mtz
                        if(results_contents[i].lastIndexOf(".mtz") == (results_contents[i].length()-4)) {
                            job.setOutputMTZ(results_contents[i]);
                        }

                        //check if the file ends in .sol,
                        //if so then it should be our output_sol
                        if(results_contents[i].lastIndexOf(".sol") == (results_contents[i].length()-4)) {
                            job.setOutputSol(results_contents[i]);
                            
                            //setup a FileReader and BufferReader for the .sol file
                            FileReader fr = new FileReader(results_dir+"/"+results_contents[i]);
                            BufferedReader in = new BufferedReader(fr);

                            String sline;
                            int linecount = 1;
                            
                            //iterate through the .sol file
                            while((sline = in.readLine()) != null) {
                                
                                //look for the line with "TFZ" and "PAK"
                                if(linecount == 3 
                                        & (sline.contains("TFZ") 
                                        & sline.contains("PAK"))) {
                                    
                                    //calculate the positions at which to substring()
                                    int z_start_pos = sline.lastIndexOf("TFZ")+4;
                                    int z_end_pos = sline.lastIndexOf("PAK");
                                    int llg_start_pos = sline.lastIndexOf("LLG")+4;
                                    
                                    //set the Zscore by substring()-ing the line.
                                    job.setZScore(Double.parseDouble(sline.substring(z_start_pos, z_end_pos).trim()));
                                    job.setLogLikelihoodGain(Double.parseDouble(sline.substring(llg_start_pos).trim()));
                                    
                                    //break out of the loop, as we only need the
                                    //top line.
                                    break;
                                }
                                linecount++;
                            }
                        }
                        
                        //check if the file ends in .sum,
                        //if so then it should be our output_sum
                        if(results_contents[i].lastIndexOf(".sum") == (results_contents[i].length()-4)) {
                            job.setOutputSum(results_contents[i]);
                        }

                    } //end of if the filename contains PDB & Spacegroup

                } //end of for loop
            } // end if results_contents !null


        } catch(Exception ex) {
            //probabaly should do something here?
            ex.printStackTrace();
        }
        
        //return the PXJob with the output files set..
        //or if we didn't get anything back, this will return the PXJob as is
        return job;
    }

    /**
     * Writes the Apple XML Plist for batch submitting a job.
     * This only writes 1 job per batch file, as submitting all jobs in 1 batch
     * file requires them all to finish before results can be got back.
     * 
     * Note: PXJob must have it's:
     * "job_name", "input_mtz", "input_pdb", "space_group",
     * "f_label", "sigf_label", "molecular_weight", 
     * "number_of_molecules_in_asymmetric_unit", 
     * "number_of_copies_of_ensemble_to_search_for", "rmsd" and "packing"
     * variables set.
     * 
     * @param job PXJob entity to write the batch submission file for
     * @param directory_path the directory that files for this submission are
     * stored in
     * @return String the location of the batch submission file
     * @throws javax.xml.parsers.ParserConfigurationException
     */
    private String writeSubmissionFileForJob(PXJob job, String directory_path) throws ParserConfigurationException {
        
        //DEBUG STATEMENT
        if(this.debug) {
            System.out.println("PXXGridController:writeSubmissionFileForJob()# BEGIN");
        }
        
        
        try {
            //DEBUG STATEMENT
            if(this.debug) {
                System.out.println("PXXGridController:writeSubmissionFileForJob()# get job name from PXJob");
            }
            
            //grab the job_name from the PXJob
            String job_name = job.getJobName();
            
            //DEBUG STATEMENT
            if(this.debug) {
                System.out.println("PXXGridController:writeSubmissionFileForJob()# Base64 encode MTZ File");
            }
            
            //Base64 encode the files (Xgrid needs them that way)
            String base64_mtz = PXUtils.base64EncodeFile(directory_path + "/" + job.getInputMTZ());
            
            //DEBUG STATEMENT
            if(this.debug) {
                System.out.println("PXXGridController:writeSubmissionFileForJob()# Base64 encode PDB File");
            }
  
            String base64_pdb = PXUtils.base64EncodeFile(directory_path + "/" + job.getInputPDB());
            
            //DEBUG STATEMENT
            if(this.debug) {
                System.out.println("PXXGridController:writeSubmissionFileForJob()# Base64 encode ruby_phaser.rb");
            }
            
            String base64_rubyphaser = PXUtils.base64EncodeFile("/mrconfig/ruby_phaser.rb");

            //DEBUG STATEMENT
            if(this.debug) {
                System.out.println("PXXGridController:writeSubmissionFileForJob()# setup XML Document");
            }
            
            //setup a Document for us to build XML into
            DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
            DocumentBuilder db = dbf.newDocumentBuilder();
            Document doc = db.newDocument();

            //DEBUG STATEMENT
            if(this.debug) {
                System.out.println("PXXGridController:writeSubmissionFileForJob()# begin constructing XML Plist");
            }
            
            //Begin the XML Plist -- let the ugliness begin!
            Element plistElement = doc.createElement("plist");
            plistElement.setAttribute("version", "1.0");
            doc.appendChild(plistElement);
            Element plistArrayElement = doc.createElement("array");
            plistElement.appendChild(plistArrayElement);
            Element plistDictElement = doc.createElement("dict");
            plistArrayElement.appendChild(plistDictElement);
            Element nameKeyElement = doc.createElement("key");
            plistDictElement.appendChild(nameKeyElement);
            Text nameKeyText = doc.createTextNode("name");
            nameKeyElement.appendChild(nameKeyText);
            Element nameStringElement = doc.createElement("string");
            plistDictElement.appendChild(nameStringElement);
            Text nameStringText = doc.createTextNode("MR Grid: " + job_name);
            nameStringElement.appendChild(nameStringText);

            //DEBUG STATEMENT
            if(this.debug) {
                System.out.println("PXXGridController:writeSubmissionFileForJob()# add inputFiles to XML Plist");
            }
            
            //create the inputFiles part of the XML Plist
            Element inputFilesKeyElement = doc.createElement("key");
            plistDictElement.appendChild(inputFilesKeyElement);
            Text inputFilesKeyText = doc.createTextNode("inputFiles");
            inputFilesKeyElement.appendChild(inputFilesKeyText);
            Element inputFilesDictElement = doc.createElement("dict");
            plistDictElement.appendChild(inputFilesDictElement);

            //DEBUG STATEMENT
            if(this.debug) {
                System.out.println("PXXGridController:writeSubmissionFileForJob()# add Ruby script to XML Plist");
            }
            
            //add the ruby_phaser.rb script as an inputFile
            Element rubyfileKeyElement = doc.createElement("key");
            inputFilesDictElement.appendChild(rubyfileKeyElement);
            Text rubyfileText = doc.createTextNode("rubyfile");
            rubyfileKeyElement.appendChild(rubyfileText);
            Element rubyfileDictElement = doc.createElement("dict");
            inputFilesDictElement.appendChild(rubyfileDictElement);
            Element rubyfileFileDataKeyElement = doc.createElement("key");
            rubyfileDictElement.appendChild(rubyfileFileDataKeyElement);
            Text rubyfileFileDataText = doc.createTextNode("fileData");
            rubyfileFileDataKeyElement.appendChild(rubyfileFileDataText);
            Element rubyfileDataElement = doc.createElement("data");
            rubyfileDictElement.appendChild(rubyfileDataElement);
            Text rubyfileDataText = doc.createTextNode(base64_rubyphaser);
            rubyfileDataElement.appendChild(rubyfileDataText);
            Element rubyfileIsExecutableKeyElement = doc.createElement("key");
            rubyfileDictElement.appendChild(rubyfileIsExecutableKeyElement);
            Text rubyfileIsExecutableKeyText = doc.createTextNode("isExecutable");
            rubyfileIsExecutableKeyElement.appendChild(rubyfileIsExecutableKeyText);
            Element rubyfileIsExecutableStringElement = doc.createElement("string");
            rubyfileDictElement.appendChild(rubyfileIsExecutableStringElement);
            Text rubyfileIsExecutableStringText = doc.createTextNode("YES");
            rubyfileIsExecutableStringElement.appendChild(rubyfileIsExecutableStringText);

            //DEBUG STATEMENT
            if(this.debug) {
                System.out.println("PXXGridController:writeSubmissionFileForJob()# add MTZ to XML Plist");
            }
            
            //add the input_mtz as an inputFile
            Element mtzfileKeyElement = doc.createElement("key");
            inputFilesDictElement.appendChild(mtzfileKeyElement);
            Text mtzfileText = doc.createTextNode("mtzfile");
            mtzfileKeyElement.appendChild(mtzfileText);
            Element mtzfileDictElement = doc.createElement("dict");
            inputFilesDictElement.appendChild(mtzfileDictElement);
            Element mtzfileFileDataKeyElement = doc.createElement("key");
            mtzfileDictElement.appendChild(mtzfileFileDataKeyElement);
            Text mtzfileFileDataText = doc.createTextNode("fileData");
            mtzfileFileDataKeyElement.appendChild(mtzfileFileDataText);
            Element mtzfileDataElement = doc.createElement("data");
            mtzfileDictElement.appendChild(mtzfileDataElement);
            Text mtzfileDataText = doc.createTextNode(base64_mtz);
            mtzfileDataElement.appendChild(mtzfileDataText);
            Element mtzfileIsExecutableKeyElement = doc.createElement("key");
            mtzfileDictElement.appendChild(mtzfileIsExecutableKeyElement);
            Text mtzfileIsExecutableKeyText = doc.createTextNode("isExecutable");
            mtzfileIsExecutableKeyElement.appendChild(mtzfileIsExecutableKeyText);
            Element mtzfileIsExecutableStringElement = doc.createElement("string");
            mtzfileDictElement.appendChild(mtzfileIsExecutableStringElement);
            Text mtzfileIsExecutableStringText = doc.createTextNode("NO");
            mtzfileIsExecutableStringElement.appendChild(mtzfileIsExecutableStringText);

            //DEBUG STATEMENT
            if(this.debug) {
                System.out.println("PXXGridController:writeSubmissionFileForJob()# add PDB to XML Plist");
            }
            
            //add the input_pdb as an inputFile
            Element pdbfileKeyElement = doc.createElement("key");
            inputFilesDictElement.appendChild(pdbfileKeyElement);
            Text pdbfileText = doc.createTextNode(job.getInputPDB());
            pdbfileKeyElement.appendChild(pdbfileText);
            Element pdbfileDictElement = doc.createElement("dict");
            inputFilesDictElement.appendChild(pdbfileDictElement);
            Element pdbfileFileDataKeyElement = doc.createElement("key");
            pdbfileDictElement.appendChild(pdbfileFileDataKeyElement);
            Text pdbfileFileDataText = doc.createTextNode("fileData");
            pdbfileFileDataKeyElement.appendChild(pdbfileFileDataText);
            Element pdbfileDataElement = doc.createElement("data");
            pdbfileDictElement.appendChild(pdbfileDataElement);
            Text pdbfileDataText = doc.createTextNode(base64_pdb);
            pdbfileDataElement.appendChild(pdbfileDataText);
            Element pdbfileIsExecutableKeyElement = doc.createElement("key");
            pdbfileDictElement.appendChild(pdbfileIsExecutableKeyElement);
            Text pdbfileIsExecutableKeyText = doc.createTextNode("isExecutable");
            pdbfileIsExecutableKeyElement.appendChild(pdbfileIsExecutableKeyText);
            Element pdbfileIsExecutableStringElement = doc.createElement("string");
            pdbfileDictElement.appendChild(pdbfileIsExecutableStringElement);
            Text pdbfileIsExecutableStringText = doc.createTextNode("NO");
            pdbfileIsExecutableStringElement.appendChild(pdbfileIsExecutableStringText);

            //DEBUG STATEMENT
            if(this.debug) {
                System.out.println("PXXGridController:writeSubmissionFileForJob()# add taskSpecifications to XML Plist");
            }
            
            //the taskSpecifications section
            Element taskSpecificationsKeyElement = doc.createElement("key");
            plistDictElement.appendChild(taskSpecificationsKeyElement);
            Text taskSpecificationsKeyText = doc.createTextNode("taskSpecifications");
            taskSpecificationsKeyElement.appendChild(taskSpecificationsKeyText);
            Element taskSpecificationsDictElement = doc.createElement("dict");
            plistDictElement.appendChild(taskSpecificationsDictElement);
            Element taskZeroKeyElement = doc.createElement("key");
            taskSpecificationsDictElement.appendChild(taskZeroKeyElement);
            Text taskZeroKeyText = doc.createTextNode("0");
            taskZeroKeyElement.appendChild(taskZeroKeyText);
            Element taskZeroDictElement = doc.createElement("dict");
            taskSpecificationsDictElement.appendChild(taskZeroDictElement);
            Element taskZeroCommandKeyElement = doc.createElement("key");
            taskZeroDictElement.appendChild(taskZeroCommandKeyElement);
            Text taskZeroCommandKeyText = doc.createTextNode("command");
            taskZeroCommandKeyElement.appendChild(taskZeroCommandKeyText);
            Element taskZeroCommandStringElement = doc.createElement("string");
            taskZeroDictElement.appendChild(taskZeroCommandStringElement);
            Text taskZeroCommandStringText = doc.createTextNode("/usr/bin/ruby");
            taskZeroCommandStringElement.appendChild(taskZeroCommandStringText);
            Element taskZeroCommandArgumentsKeyElement = doc.createElement("key");
            taskZeroDictElement.appendChild(taskZeroCommandArgumentsKeyElement);
            Text taskZeroCommandArgumentsKeyText = doc.createTextNode("arguments");
            taskZeroCommandArgumentsKeyElement.appendChild(taskZeroCommandArgumentsKeyText);
            Element taskZeroCommandArgumentsArrayElement = doc.createElement("array");
            taskZeroDictElement.appendChild(taskZeroCommandArgumentsArrayElement);
            Element taskZeroArgumentsRubyphaserStringElement = doc.createElement("string");
            taskZeroCommandArgumentsArrayElement.appendChild(taskZeroArgumentsRubyphaserStringElement);
            Text taskZeroArgumentsRubyphaserStringText = doc.createTextNode("ruby_phaser.rb");
            taskZeroArgumentsRubyphaserStringElement.appendChild(taskZeroArgumentsRubyphaserStringText);
            Element taskZeroArgumentsMTZStringElement = doc.createElement("string");
            taskZeroCommandArgumentsArrayElement.appendChild(taskZeroArgumentsMTZStringElement);
            Text taskZeroArgumentsMTZStringText = doc.createTextNode("mtzfile.mtz");
            taskZeroArgumentsMTZStringElement.appendChild(taskZeroArgumentsMTZStringText);
            Element taskZeroArgumentsPDBStringElement = doc.createElement("string");
            taskZeroCommandArgumentsArrayElement.appendChild(taskZeroArgumentsPDBStringElement);
            Text taskZeroArgumentsPDBStringText = doc.createTextNode(job.getInputPDB());
            taskZeroArgumentsPDBStringElement.appendChild(taskZeroArgumentsPDBStringText);
            Element taskZeroArgumentsFStringElement = doc.createElement("string");
            taskZeroCommandArgumentsArrayElement.appendChild(taskZeroArgumentsFStringElement);
            Text taskZeroArgumentsFStringText = doc.createTextNode(job.getFLabel());
            taskZeroArgumentsFStringElement.appendChild(taskZeroArgumentsFStringText);
            Element taskZeroArgumentsSigFStringElement = doc.createElement("string");
            taskZeroCommandArgumentsArrayElement.appendChild(taskZeroArgumentsSigFStringElement);
            Text taskZeroArgumentsSigFStringText = doc.createTextNode(job.getSigfLabel());
            taskZeroArgumentsSigFStringElement.appendChild(taskZeroArgumentsSigFStringText);
            Element taskZeroArgumentsSGStringElement = doc.createElement("string");
            taskZeroCommandArgumentsArrayElement.appendChild(taskZeroArgumentsSGStringElement);
            Text taskZeroArgumentsSGStringText = doc.createTextNode(job.getSpaceGroup());
            taskZeroArgumentsSGStringElement.appendChild(taskZeroArgumentsSGStringText);
            Element taskZeroArgumentsMWStringElement = doc.createElement("string");
            taskZeroCommandArgumentsArrayElement.appendChild(taskZeroArgumentsMWStringElement);
            Text taskZeroArgumentsMWStringText = doc.createTextNode(String.valueOf(job.getMolecularWeight()));
            taskZeroArgumentsMWStringElement.appendChild(taskZeroArgumentsMWStringText);
            Element taskZeroArgumentsAUStringElement = doc.createElement("string");
            taskZeroCommandArgumentsArrayElement.appendChild(taskZeroArgumentsAUStringElement);
            Text taskZeroArgumentsAUStringText = doc.createTextNode(String.valueOf(job.getNumberOfMoleculesInAsymmetricUnit()));
            taskZeroArgumentsAUStringElement.appendChild(taskZeroArgumentsAUStringText);
            Element taskZeroArgumentsRMSDStringElement = doc.createElement("string");
            taskZeroCommandArgumentsArrayElement.appendChild(taskZeroArgumentsRMSDStringElement);
            Text taskZeroArgumentsRMSDStringText = doc.createTextNode(String.valueOf(job.getRMSD()));
            taskZeroArgumentsRMSDStringElement.appendChild(taskZeroArgumentsRMSDStringText);
            Element taskZeroArgumentsEnsembleStringElement = doc.createElement("string");
            taskZeroCommandArgumentsArrayElement.appendChild(taskZeroArgumentsEnsembleStringElement);
            Text taskZeroArgumentsEnsembleStringText = doc.createTextNode(String.valueOf(job.getNumberOfCopiesOfEnsembleToSearchFor()));
            taskZeroArgumentsEnsembleStringElement.appendChild(taskZeroArgumentsEnsembleStringText);
            Element taskZeroArgumentsPackingStringElement = doc.createElement("string");
            taskZeroCommandArgumentsArrayElement.appendChild(taskZeroArgumentsPackingStringElement);
            Text taskZeroArgumentsPackingStringText = doc.createTextNode(String.valueOf(job.getPacking()));
            taskZeroArgumentsPackingStringElement.appendChild(taskZeroArgumentsPackingStringText);

            //DEBUG STATEMENT
            if(this.debug) {
                System.out.println("PXXGridController:writeSubmissionFileForJob()# add inputFileMap to XML Plist");
            }
            
            //the inputFileMap session
            Element taskInputFileMapKeyElement = doc.createElement("key");
            taskZeroDictElement.appendChild(taskInputFileMapKeyElement);
            Text taskInputFileMapKeyText = doc.createTextNode("inputFileMap");
            taskInputFileMapKeyElement.appendChild(taskInputFileMapKeyText);
            Element taskInputFileMapDictElement = doc.createElement("dict");
            taskZeroDictElement.appendChild(taskInputFileMapDictElement);
            Element IFMRubyKeyElement = doc.createElement("key");
            taskInputFileMapDictElement.appendChild(IFMRubyKeyElement);
            Text IFMRubyKeyText = doc.createTextNode("ruby_phaser.rb");
            IFMRubyKeyElement.appendChild(IFMRubyKeyText);
            Element IFMRubyStringElement = doc.createElement("string");
            taskInputFileMapDictElement.appendChild(IFMRubyStringElement);
            Text IFMRubyStringText = doc.createTextNode("rubyfile");
            IFMRubyStringElement.appendChild(IFMRubyStringText);
            Element IFMMTZKeyElement = doc.createElement("key");
            taskInputFileMapDictElement.appendChild(IFMMTZKeyElement);
            Text IFMMTZKeyText = doc.createTextNode("mtzfile.mtz");
            IFMMTZKeyElement.appendChild(IFMMTZKeyText);
            Element IFMMTZStringElement = doc.createElement("string");
            taskInputFileMapDictElement.appendChild(IFMMTZStringElement);
            Text IFMMTZStringText = doc.createTextNode("mtzfile");
            IFMMTZStringElement.appendChild(IFMMTZStringText);
            Element IFMPDBKeyElement = doc.createElement("key");
            taskInputFileMapDictElement.appendChild(IFMPDBKeyElement);
            Text IFMPDBKeyText = doc.createTextNode(job.getInputPDB());
            IFMPDBKeyElement.appendChild(IFMPDBKeyText);
            Element IFMPDBStringElement = doc.createElement("string");
            taskInputFileMapDictElement.appendChild(IFMPDBStringElement);
            Text IFMPDBStringText = doc.createTextNode(job.getInputPDB());
            IFMPDBStringElement.appendChild(IFMPDBStringText);
            
            //DEBUG STATEMENT
            if(this.debug) {
                System.out.println("PXXGridController:writeSubmissionFileForJob()# transform the XML Plist");
            }
            
            //create a Transformer and transform the xml doc into an actual file
            Transformer optimus_prime = TransformerFactory.newInstance().newTransformer();
            String xml_file = directory_path + "/" + job_name + ".xml";
            
            File XMLFile = new File(xml_file);
            //insert transformer sound effect here ;p
            optimus_prime.transform(new DOMSource(doc), new StreamResult(XMLFile));

            //DEBUG STATEMENT
            if(this.debug) {
                System.out.println("PXXGridController:writeSubmissionFileForJob()# END");
            }
            
            //return the path of the file
            return xml_file;
            
        } catch (TransformerException ex) {
            //if it all goes horribly wrong, return "false"
            ex.printStackTrace();
            return "false";
        }
    }

    /**
     * Submits an Apple XML Plist file to a grid as a batch submission.
     * 
     * @param submission_file the location of the submission file
     * @param grid_name the display_name of the grid to submit to
     * @return int the job_id assigned to the job by the grid.
     */
    private int submitSubmissionFileToGrid(String submission_file, String grid_name) {
        
        //DEBUG STATEMENT
        if(this.debug) {
            System.out.println("PXXGridController:submitSubmissionFileToGrid()# BEGIN");
        }
        
        //DEBUG STATEMENT
        if(this.debug) {
            System.out.println("PXXGridController:submitSubmissionFileToGrid()# parse the Grid map");
        }
        
        //parse the xgrid_definitions.xml file into a Map
        Map gridMap = PXUtils.parseGridXML("/mrconfig/xgrid_definitions.xml");
        
        //DEBUG STATEMENT
        if(this.debug) {
            System.out.println("PXXGridController:submitSubmissionFileToGrid()# create PXGrid entity");
        }
        
        //convert gridMap entry into a PXGrid entity.
        PXGrid grid = (PXGrid) gridMap.get(grid_name);

        //DEBUG STATEMENT
        if(this.debug) {
            System.out.println("PXXGridController:submitSubmissionFileToGrid()# grab variables from PXGrid");
        }
        
        //get the various attributes required to submit to the grid
        String host = grid.getHost();
        String auth_method = grid.getAuth_method();
        String password = grid.getPassword();
        int grid_id = grid.getGridID();

        //DEBUG STATEMENT
        if(this.debug) {
            System.out.println("PXXGridController:submitSubmissionFileToGrid()# formulate grid submission command");
        }
        
        //formulate the command string
        String cmd = "xgrid -h " + host + " -a " + auth_method;

        //if we're authenticating by password, then use it
        if (auth_method.contains("password")) {
            cmd += " -p " + password;
        }

        //add the other stuff to the cmd string
        cmd += " -gid " + grid_id + " -job batch " + submission_file;
        
        try {
            
            //DEBUG STATEMENT
            if(this.debug) {
                System.out.println("PXXGridController:submitSubmissionFileToGrid()# create a runtime process");
            }
            
            //create a Process from the cmd
            Process prcs = Runtime.getRuntime().exec(cmd);
            InputStream cmd_output = prcs.getInputStream();

            //DEBUG STATEMENT
            if(this.debug) {
                System.out.println("PXXGridController:submitSubmissionFileToGrid()# wait for process to end");
            }
            
            //wait for the process to complete
            int exit_val = prcs.waitFor();

            //DEBUG STATEMENT
            if(this.debug) {
                System.out.println("PXXGridController:submitSubmissionFileToGrid()# process ended, check exit value");
            }
            
            //check it exited right (exit_val == 0)
            if (exit_val == 0) {
                
                //DEBUG STATEMENT
                if(this.debug) {
                    System.out.println("PXXGridController:submitSubmissionFileToGrid()# read the cmd_output");
                }
                
                //put the InputStream into a BufferedReader
                BufferedReader br = new BufferedReader(new InputStreamReader(cmd_output));
                StringBuilder sb = new StringBuilder();
                String line = null;

                //DEBUG STATEMENT
                if(this.debug) {
                    System.out.println("PXXGridController:submitSubmissionFileToGrid()# dump output to String");
                }
                
                //iterate through the BufferedReader stream and append to string
                while ((line = br.readLine()) != null) {
                    sb.append(line + "\n");
                }
                
                //close the BufferedReader
                br.close();
                
                //DEBUG STATEMENT
                if(this.debug) {
                    System.out.println("PXXGridController:submitSubmissionFileToGrid()# try to set the value from output");
                }
                
                //convert the StringBuilder to String, then substring() it to get
                //the job_id assigned by the grid.
                String sb_string = sb.toString();
                //hopefully this way is tiger-happy?
                int begin_split = sb_string.indexOf("=")+1; //this way we get after the =
                int end_split = sb_string.indexOf(";");
                
                //System.out.println("cmd stdout: #"+sb_string+"#");
                //System.out.println("begin at "+begin_split+", end at "+end_split);
                String a = sb_string.substring(begin_split, end_split).trim();
                //System.out.println("theoretically that gives us: "+a);
                
                
                return Integer.parseInt(a);
            }
            
        } catch(Exception e) {
            e.printStackTrace(); 
        }

        //returning minus 1 to indicate error.
        //theoretically you'll never get a job_id of minus 1 back from the Xgrid.
        //If you do, I'd be very worried.
        return -1;
    }
    
    //THIS METHOD IS NOT YET USED, IT'S HERE FOR FUTURE RELEASES
    
    /**
     * Performs an action on the job.
     * Note: action must be one of: stop, suspend, resume, delete, restart.
     * 
     * @param action String the action to perform on the job
     * @param job PXJob the job to perform it on
     * @return Boolean whether the action was performed or not
     * @throws java.lang.Exception thrown if the action is not a valid action
     */
    @Override
    public Boolean performGridActionOnJob(String action, PXJob job) throws Exception {

        if(job.getJobName() == null) {
            //throw new Tantrum("job_name not set for PXJob ");
            throw new Exception("job_name not set for PXJob ");
        }

        if(job.getGridController() == null) {
            throw new Exception("grid_controller not set for PXJob ");
        }
        
        
        ArrayList valid_actions = new ArrayList();
        valid_actions.add("stop");
        valid_actions.add("suspend");
        valid_actions.add("resume");
        valid_actions.add("delete");
        valid_actions.add("restart");
        
        if(!valid_actions.contains(action)) {
            throw new Exception("this action cannot be enacted on the grid");
        }
        
        
        //parse the xgrid_definitions.xml file into a Map
        Map gridMap = PXUtils.parseGridXML("/mrconfig/xgrid_definitions.xml");
        
        //grab the grid_job_id and grid_controller from the PXJob
        int job_id = job.getGridJobID();
        String grid_name = job.getGridController();
        
        //convert gridMap entry into a PXGrid entity.
        PXGrid grid = (PXGrid) gridMap.get(grid_name);

        //get the various attributes required to submit to the grid
        String host = grid.getHost();
        String auth_method = grid.getAuth_method();
        String password = grid.getPassword();
        int grid_id = grid.getGridID();
        
        //formulate the command string
        String cmd = "xgrid -h " + host + " -a " + auth_method;

        //if we're authenticating by password, then use it
        if (auth_method.contains("password")) {
            cmd += " -p " + password;
        }

        //add the other stuff to the cmd string
        cmd += " -gid " + grid_id + " -job "+action+" -id " + job_id;
        
        try {
            
            //DEBUG STATEMENT
            if(this.debug) {
                System.out.println("PXXGridController:submitSubmissionFileToGrid()# create a runtime process");
            }
            
            //create a Process from the cmd
            Process prcs = Runtime.getRuntime().exec(cmd);
            InputStream cmd_output = prcs.getInputStream();

            //DEBUG STATEMENT
            if(this.debug) {
                System.out.println("PXXGridController:submitSubmissionFileToGrid()# wait for process to end");
            }
            
            //wait for the process to complete
            int exit_val = prcs.waitFor();

            //DEBUG STATEMENT
            if(this.debug) {
                System.out.println("PXXGridController:submitSubmissionFileToGrid()# process ended, check exit value");
            }
            
            //check it exited right (exit_val == 0) then theoretically it worked?
            if (exit_val == 0) {
                return true;
            }
            
        } catch (Exception ex) {
            ex.printStackTrace(); 
        }
        
        return false;
    }
}
